import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.metrics import mean_squared_error
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
from sklearn.preprocessing import LabelEncoder, StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.impute import SimpleImputer

# 1. 数据加载与初步分析
train = pd.read_csv('C:/PSSENDFILES/train.csv')
test = pd.read_csv('C:/PSSENDFILES/test.csv')

print(f"训练集形状: {train.shape}")
print(f"测试集形状: {test.shape}")

# 2. 数据预处理
# 合并数据集以便统一处理
all_data = pd.concat([train.drop('SalePrice', axis=1), test], axis=0)

# 删除高缺失率特征
all_data.drop(['PoolQC', 'MiscFeature', 'Alley', 'Fence'], axis=1, inplace=True)

# 处理缺失值
# 数值型特征用中位数填充
num_cols = all_data.select_dtypes(include=['int64', 'float64']).columns
for col in num_cols:
    all_data[col] = all_data[col].fillna(all_data[col].median())

# 类别型特征用众数填充
cat_cols = all_data.select_dtypes(include=['object']).columns
for col in cat_cols:
    all_data[col] = all_data[col].fillna(all_data[col].mode()[0])

# 3. 特征工程
# 创建新特征
all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
all_data['TotalBath'] = all_data['FullBath'] + 0.5 * all_data['HalfBath']
all_data['Age'] = all_data['YrSold'] - all_data['YearBuilt']
all_data['IsRemodeled'] = (all_data['YearRemodAdd'] != all_data['YearBuilt']).astype(int)

# 对偏态数值特征进行对数变换
skewed_cols = ['LotFrontage', 'LotArea', 'MasVnrArea', 'BsmtFinSF1', 'BsmtFinSF2',
               'BsmtUnfSF', 'TotalBsmtSF', '1stFlrSF', '2ndFlrSF', 'GrLivArea',
               'WoodDeckSF', 'OpenPorchSF', 'EnclosedPorch', 'ScreenPorch', 'PoolArea']

for col in skewed_cols:
    if col in all_data.columns:
        all_data[col] = np.log1p(all_data[col])

# 对类别变量进行标签编码（比独热编码更节省内存）
le = LabelEncoder()
for col in cat_cols:
    if col in all_data.columns:
        all_data[col] = le.fit_transform(all_data[col].astype(str))

# 4. 重新分割数据集
X_train = all_data[:len(train)]
X_test = all_data[len(train):]
y_train = np.log1p(train['SalePrice'])  # 对目标变量进行对数变换

# 5. 模型训练与评估
# 划分验证集
X_train, X_val, y_train, y_val = train_test_split(
    X_train, y_train, test_size=0.2, random_state=42)

# 定义预处理步骤
num_pipeline = Pipeline([
    ('imputer', SimpleImputer(strategy='median')),
    ('scaler', StandardScaler())
])

cat_pipeline = Pipeline([
    ('imputer', SimpleImputer(strategy='most_frequent')),
    ('encoder', LabelEncoder())
])

preprocessor = ColumnTransformer([
    ('num', num_pipeline, num_cols),
    ('cat', cat_pipeline, cat_cols)
])

# 尝试不同模型
models = {
    'Random Forest': RandomForestRegressor(n_estimators=300, random_state=42),
    'XGBoost': XGBRegressor(n_estimators=1000, learning_rate=0.01, random_state=42, n_jobs=-1),
    'LightGBM': LGBMRegressor(n_estimators=1000, learning_rate=0.01, random_state=42, n_jobs=-1),
    'Gradient Boosting': GradientBoostingRegressor(n_estimators=300, random_state=42)
}

for name, model in models.items():
    pipeline = Pipeline([('preprocessor', preprocessor), ('model', model)])
    pipeline.fit(X_train, y_train)
    val_pred = pipeline.predict(X_val)
    rmse = np.sqrt(mean_squared_error(y_val, val_pred))
    print(f"{name} Validation RMSE: {rmse:.4f}")

# 交叉验证评估模型性能
best_model = LGBMRegressor(n_estimators=1000, learning_rate=0.01, random_state=42, n_jobs=-1)
pipeline = Pipeline([('preprocessor', preprocessor), ('model', best_model)])
cv_scores = cross_val_score(pipeline, X_train, y_train, cv=5, scoring='neg_mean_squared_error')
print(f"LightGBM Cross-Validation RMSE: {np.sqrt(-cv_scores).mean():.4f}")

# 选择表现最好的模型进行全量训练
final_pipeline = Pipeline([('preprocessor', preprocessor), ('model', best_model)])
final_pipeline.fit(X_train, y_train)

# 6. 生成预测结果
test_pred = final_pipeline.predict(X_test)
test_pred = np.expm1(test_pred)  # 转换回原始尺度

# 7. 准备提交文件
submission = pd.DataFrame({
    'Id': test['Id'],
    'SalePrice': test_pred
})
submission.to_csv('C:/PSSENDFILES/submission.csv', index=False)

print("提交文件已保存至 C:/PSSENDFILES/submission.csv")
print(submission.head())